首先创建一个SimpleHttpOperator
http_test_operator = SimpleHttpOperator(
http_coon_id='http_test_1',
endpoint='https://free-api.heweather.com/s6/weather/forecast?unit=m&location=北京&lang=zh&key=*******************',
method='GET',
response_check=http_resp_check,
)在测试的时候发现问题如下：
[2018-04-27 11:58:31,880] {http_hook.py:77} INFO - Sending 'GET' to url: https://www.google.com/s6/weather/forecast?unit=m&location=朝阳,北京&lang=zh&key=*********很明显url和我们想的不一样，查询官方文档得到如下解释：
Airflow还能够通过操作系统的环境变量来引用连接。环境变量需要加上前缀AIRFLOW_CONN_才能被视为连接。在Airflow管道中引用连接时，conn_id应该是没有前缀的变量的名称。例如，如果conn_id 名为postgres_master环境变量应该被命名 AIRFLOW_CONN_POSTGRES_MASTER（注意环境变量必须全部为大写）。Airflow假定从环境变量返回的值是URI格式（例如 postgres://user:password@localhost:5432/master或s3://accesskey:secretkey@S3）。airflow会首先根据conn_id在环境变量中寻找对应的host，如果没有则使用默认的，数据库中connection表的http_default的host值：https://www.google.com/。
于是先设置环境变量吧
export AIRFLOW_CONN_WEATHER=https://free-api.heweather.com/并修改代码中endpoint参数如下：
http_test_operator = SimpleHttpOperator(
http_coon_id='http_test_1',
endpoint='/s6/weather/forecast?unit=m&location=北京&lang=zh&key=*******************',
method='GET',
response_check=http_resp_check,
)继续测试，发现调试信息如下：
[2018-04-27 12:21:13,131] {http_hook.py:77} INFO - Sending 'GET' to url: http://free-api.heweather.com/s6/weather/forecast?unit=m&location=北京&lang=zh&key=********************查看源码
class HttpHook(BaseHook):
def get_conn(self, headers):
"""
Returns http session for use with requests
"""
conn = self.get_connection(self.http_conn_id)
session = requests.Session()
if "://" in conn.host:
self.base_url = conn.host
else:
# schema defaults to HTTP
schema = conn.schema if conn.schema else "http"
self.base_url = schema + "://" + conn.host实际请求的正是这个self.base_url，那么这个conn.schema是哪来的呢？
class BaseHook(LoggingMixin):
@classmethod
def get_connections(cls, conn_id):
conn = cls._get_connection_from_env(conn_id)
if conn:
conns = [conn]
else:
conns = cls._get_connections_from_db(conn_id)
return conns其实到这里已经看到我们的环境变量是怎么起作用了，也看到了如果环境变量没有设置，是会从数据库中选择的。在数据库中connection表的schema字段正是我们要找的。
@classmethod
def _get_connection_from_env(cls, conn_id):
environment_uri = os.environ.get(CONN_ENV_PREFIX + conn_id.upper())
conn = None
if environment_uri:
conn = Connection(conn_id=conn_id, uri=environment_uri)
return conn在这个地方，我们明显可以看到官方文档中的描述（环境变量的设置）是如何被执行的。
class Connection(Base, LoggingMixin):
def __init__(
self, conn_id=None, conn_type=None,
host=None, login=None, password=None,
schema=None, port=None, extra=None,
uri=None):
self.conn_id = conn_id
if uri:
self.parse_from_uri(uri)
else:
self.conn_type = conn_type
self.host = host
self.login = login
self.password = password
self.schema = schema
self.port = port
self.extra = extra
def parse_from_uri(self, uri):
temp_uri = urlparse(uri)
hostname = temp_uri.hostname or ''
if '%2f' in hostname:
hostname = hostname.replace('%2f', '/').replace('%2F', '/')
conn_type = temp_uri.scheme
if conn_type == 'postgresql':
conn_type = 'postgres'
self.conn_type = conn_type
self.host = hostname
self.schema = temp_uri.path[1:]
self.login = temp_uri.username
self.password = temp_uri.password
self.port = temp_uri.port到这里基本上就明白了schema是哪里来的
self.schema = temp_uri.path[1:]这是什么意思？难道我要写成"https://free-api.heweather.com/https"
各位看官，到这里基本上我们就明白问题所在了
你可以修改这里的源码为
self.schema = conn_type进行测试。
也可以写成
环境变量也可以设置为"https://free-api.heweather.com/https"
如果你的airflow版本和我的不同，源码可能是下面这个样子：
# headers is required to make it required
def get_conn(self, headers):
"""
Returns http session for use with requests
"""
conn = self.get_connection(self.http_conn_id)
session = requests.Session()
self.base_url = conn.host
if not self.base_url.startswith('http'):
self.base_url = 'http://' + self.base_url
if conn.port:
self.base_url = self.base_url + ":" + str(conn.port) + "/"
if conn.login:
session.auth = (conn.login, conn.password)
if headers:
session.headers.update(headers)
return session关键在这里
self.base_url = conn.host
if not self.base_url.startswith('http'):
self.base_url = 'http://' + self.base_url这里的判断条件（not self.base_url.startswith('http')）必然为真，所以只会拼接“http://”（头大）
除了更改环境变量或者上述代码还要更改代码如下：
def get_conn(self, headers):
"""
Returns http session for use with requests
"""
conn = self.get_connection(self.http_conn_id)
session = requests.Session()
self.base_url = '{}://{}'.format(conn.sharem, conn.host)
# if not self.base_url.startswith('http'):
# self.base_url = 'http://' + self.base_url
if conn.port:
self.base_url = self.base_url + ":" + str(conn.port) + "/"
if conn.login:
session.auth = (conn.login, conn.password)
if headers:
session.headers.update(headers)
return session注意：这样是没有添加证书验证的！
运行会警告：
/usr/local/lib/python3.6/site-packages/urllib3/connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings运行结果：
{'HeWeather6': [{'basic': {'cid': 'CN101010300', 'location': '朝阳', 'parent_city': '北京', 'admin_area': '北京', 'cnty': '中国', 'lat': '39.92148972', 'lon': '116.48641205', 'tz': '+8.00'}, 'update': {'loc': '2018-04-27 17:47', 'utc': '2018-04-27 09:47'}, 'status': 'ok', 'daily_forecast': [{'cond_code_d': '100', 'cond_code_n': '100', 'cond_txt_d': '晴', 'cond_txt_n': '晴', 'date': '2018-04-27', 'hum': '40', 'mr': '16:06', 'ms': '04:04', 'pcpn': '0.0', 'pop': '0', 'pres': '1021', 'sr': '05:20', 'ss': '19:04', 'tmp_max': '27', 'tmp_min': '13', 'uv_index': '7', 'vis': '10', 'wind_deg': '0', 'wind_dir': '无持续风向', 'wind_sc': '1-2', 'wind_spd': '7'}, {'cond_code_d': '100', 'cond_code_n': '100', 'cond_txt_d': '晴', 'cond_txt_n': '晴', 'date': '2018-04-28', 'hum': '41', 'mr': '17:11', 'ms': '04:35', 'pcpn': '0.0', 'pop': '0', 'pres': '1013', 'sr': '05:18', 'ss': '19:05', 'tmp_max': '27', 'tmp_min': '16', 'uv_index': '8', 'vis': '20', 'wind_deg': '188', 'wind_dir': '南风', 'wind_sc': '1-2', 'wind_spd': '7'}, {'cond_code_d': '101', 'cond_code_n': '305', 'cond_txt_d': '多云', 'cond_txt_n': '小雨', 'date': '2018-04-29', 'hum': '43', 'mr': '18:15', 'ms': '05:07', 'pcpn': '0.0', 'pop': '0', 'pres': '1005', 'sr': '05:17', 'ss': '19:06', 'tmp_max': '30', 'tmp_min': '18', 'uv_index': '8', 'vis': '20', 'wind_deg': '181', 'wind_dir': '南风', 'wind_sc': '1-2', 'wind_spd': '10'}]}]}
